{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression Classifier for Text with Iterative Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Requires fewer computate resource when used with massive datasets compared to [log-reg.ipynb](log-reg.ipynb)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Obtaining the IMDb movie review dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import tarfile\n",
    "import time\n",
    "import urllib.request\n",
    "\n",
    "source = 'http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz'\n",
    "target = 'aclImdb_v1.tar.gz'\n",
    "\n",
    "if os.path.exists(target):\n",
    "    os.remove(target)\n",
    "\n",
    "def reporthook(count, block_size, total_size):\n",
    "    global start_time\n",
    "    if count == 0:\n",
    "        start_time = time.time()\n",
    "        return\n",
    "    duration = time.time() - start_time\n",
    "    progress_size = int(count * block_size)\n",
    "    speed = progress_size / (1024.**2 * duration)\n",
    "    percent = count * block_size * 100. / total_size\n",
    "\n",
    "    sys.stdout.write(f'\\r{int(percent)}% | {progress_size / (1024.**2):.2f} MB '\n",
    "                     f'| {speed:.2f} MB/s | {duration:.2f} sec elapsed')\n",
    "    sys.stdout.flush()\n",
    "\n",
    "\n",
    "if not os.path.isdir('aclImdb') and not os.path.isfile('aclImdb_v1.tar.gz'):\n",
    "    urllib.request.urlretrieve(source, target, reporthook)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.isdir('aclImdb'):\n",
    "\n",
    "    with tarfile.open(target, 'r:gz') as tar:\n",
    "        tar.extractall()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing the movie dataset into more convenient format"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Install pyprind by uncommenting the next code cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install pyprind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0% [##############################] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:19\n"
     ]
    }
   ],
   "source": [
    "import pyprind\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "from packaging import version\n",
    "\n",
    "\n",
    "# change the `basepath` to the directory of the\n",
    "# unzipped movie dataset\n",
    "\n",
    "basepath = 'aclImdb'\n",
    "\n",
    "labels = {'pos': 1, 'neg': 0}\n",
    "\n",
    "# if the progress bar does not show, change stream=sys.stdout to stream=2\n",
    "pbar = pyprind.ProgBar(50000, stream=sys.stdout)\n",
    "\n",
    "df = pd.DataFrame()\n",
    "for s in ('test', 'train'):\n",
    "    for l in ('pos', 'neg'):\n",
    "        path = os.path.join(basepath, s, l)\n",
    "        for file in sorted(os.listdir(path)):\n",
    "            with open(os.path.join(path, file), \n",
    "                      'r', encoding='utf-8') as infile:\n",
    "                txt = infile.read()\n",
    "                \n",
    "            if version.parse(pd.__version__) >= version.parse(\"1.3.2\"):\n",
    "                x = pd.DataFrame([[txt, labels[l]]], columns=['review', 'sentiment'])\n",
    "                df = pd.concat([df, x], ignore_index=False)\n",
    "\n",
    "            else:\n",
    "                df = df.append([[txt, labels[l]]], \n",
    "                               ignore_index=True)\n",
    "            pbar.update()\n",
    "df.columns = ['review', 'sentiment']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Shuffling the DataFrame:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "if version.parse(pd.__version__) >= version.parse(\"1.3.2\"):\n",
    "    df = df.sample(frac=1, random_state=0).reset_index(drop=True)\n",
    "    \n",
    "else:\n",
    "    np.random.seed(0)\n",
    "    df = df.reindex(np.random.permutation(df.index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Optional: Saving the assembled data as CSV file:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('movie_data.csv', index=False, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>In 1974, the teenager Martha Moxley (Maggie Gr...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>OK... so... I really like Kris Kristofferson a...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>***SPOILER*** Do not read this, if you think a...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review  sentiment\n",
       "0  In 1974, the teenager Martha Moxley (Maggie Gr...          1\n",
       "1  OK... so... I really like Kris Kristofferson a...          0\n",
       "2  ***SPOILER*** Do not read this, if you think a...          0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('movie_data.csv', encoding='utf-8')\n",
    "\n",
    "# the following is necessary on some computers:\n",
    "df = df.rename(columns={\"0\": \"review\", \"1\": \"sentiment\"})\n",
    "\n",
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 2)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training a logistic regression model for document classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Working with bigger data - online algorithms and out-of-core learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This cell is not contained in the book but\n",
    "# added for convenience so that the notebook\n",
    "# can be executed starting here, without\n",
    "# executing prior code in this notebook\n",
    "\n",
    "import os\n",
    "import gzip\n",
    "\n",
    "\n",
    "if not os.path.isfile('movie_data.csv'):\n",
    "    if not os.path.isfile('movie_data.csv.gz'):\n",
    "        print('Please place a copy of the movie_data.csv.gz'\n",
    "              'in this directory. You can obtain it by'\n",
    "              'a) executing the code in the beginning of this'\n",
    "              'notebook or b) by downloading it from GitHub:'\n",
    "              'https://github.com/rasbt/machine-learning-book/'\n",
    "              'blob/main/ch08/movie_data.csv.gz')\n",
    "    else:\n",
    "        with gzip.open('movie_data.csv.gz', 'rb') as in_f, \\\n",
    "                open('movie_data.csv', 'wb') as out_f:\n",
    "            out_f.write(in_f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "\n",
    "# The `stop` is defined as earlier in this chapter\n",
    "# Added it here for convenience, so that this section\n",
    "# can be run as standalone without executing prior code\n",
    "# in the directory\n",
    "stop = stopwords.words('english')\n",
    "\n",
    "\n",
    "def tokenizer(text):\n",
    "    text = re.sub('<[^>]*>', '', text)\n",
    "    emoticons = re.findall('(?::|;|=)(?:-)?(?:\\)|\\(|D|P)', text)\n",
    "    text = re.sub('[\\W]+', ' ', text.lower()) +\\\n",
    "        ' '.join(emoticons).replace('-', '')\n",
    "    tokenized = [w for w in text.split() if w not in stop]\n",
    "    return tokenized\n",
    "\n",
    "\n",
    "def stream_docs(path):\n",
    "    with open(path, 'r', encoding='utf-8') as csv:\n",
    "        next(csv)  # skip header\n",
    "        for line in csv:\n",
    "            text, label = line[:-3], int(line[-2])\n",
    "            yield text, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('\"In 1974, the teenager Martha Moxley (Maggie Grace) moves to the high-class area of Belle Haven, Greenwich, Connecticut. On the Mischief Night, eve of Halloween, she was murdered in the backyard of her house and her murder remained unsolved. Twenty-two years later, the writer Mark Fuhrman (Christopher Meloni), who is a former LA detective that has fallen in disgrace for perjury in O.J. Simpson trial and moved to Idaho, decides to investigate the case with his partner Stephen Weeks (Andrew Mitchell) with the purpose of writing a book. The locals squirm and do not welcome them, but with the support of the retired detective Steve Carroll (Robert Forster) that was in charge of the investigation in the 70\\'s, they discover the criminal and a net of power and money to cover the murder.<br /><br />\"\"Murder in Greenwich\"\" is a good TV movie, with the true story of a murder of a fifteen years old girl that was committed by a wealthy teenager whose mother was a Kennedy. The powerful and rich family used their influence to cover the murder for more than twenty years. However, a snoopy detective and convicted perjurer in disgrace was able to disclose how the hideous crime was committed. The screenplay shows the investigation of Mark and the last days of Martha in parallel, but there is a lack of the emotion in the dramatization. My vote is seven.<br /><br />Title (Brazil): Not Available\"',\n",
       " 1)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(stream_docs(path='movie_data.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_minibatch(doc_stream, size):\n",
    "    docs, y = [], []\n",
    "    try:\n",
    "        for _ in range(size):\n",
    "            text, label = next(doc_stream)\n",
    "            docs.append(text)\n",
    "            y.append(label)\n",
    "    except StopIteration:\n",
    "        return None, None\n",
    "    return docs, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import HashingVectorizer\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "\n",
    "vect = HashingVectorizer(decode_error='ignore', \n",
    "                         n_features=2**21,\n",
    "                         preprocessor=None, \n",
    "                         tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from distutils.version import LooseVersion as Version\n",
    "from sklearn import __version__ as sklearn_version\n",
    "\n",
    "clf = SGDClassifier(loss='log_loss', random_state=1)\n",
    "\n",
    "\n",
    "doc_stream = stream_docs(path='movie_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [##############################] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:17\n"
     ]
    }
   ],
   "source": [
    "import pyprind\n",
    "pbar = pyprind.ProgBar(45)\n",
    "\n",
    "classes = np.array([0, 1])\n",
    "for _ in range(45):\n",
    "    X_train, y_train = get_minibatch(doc_stream, size=1000)\n",
    "    if not X_train:\n",
    "        break\n",
    "    X_train = vect.transform(X_train)\n",
    "    clf.partial_fit(X_train, y_train, classes=classes)\n",
    "    pbar.update()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.868\n"
     ]
    }
   ],
   "source": [
    "X_test, y_test = get_minibatch(doc_stream, size=5000)\n",
    "X_test = vect.transform(X_test)\n",
    "print(f'Accuracy: {clf.score(X_test, y_test):.3f}')"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
